# coding:utf-8
import requests
import json
import os,sys
from tkafka import ConsumerHandler
import time


def data_format(data):
    ret = {}
    for key in ['dataid', 'pkey', 'uuid', 'iid', 'url', 'classes', 'ctime', 'docType', 'outlook']:
        tmp = data.get(key)
        if not tmp:     continue
        ret[key] = tmp
    ret['dataid'] = data.get('dataid')
    ret['title'] = data['title'] if 'title' in data else data.get('title_ori', '')
    group_id = data['group_id']
    ret['group_id_online'] = group_id
    ret['iid'] = data['iid_md5'] if 'iid_md5' in data else data.get('iid','')
    keywords_list = data.get('keywords_list', [])
    if keywords_list:
        keywords_list = [{"word":w} for w in keywords_list]
    ret['keywords'] = keywords_list
    ret['content'] = ret['title']
    return ret


def fp(data):
    headers = {'Content-Type': 'application/json','Connection': 'close'}
    try:
        response = requests.post(url='http://10.39.14.233:3555/group_id/', headers=headers, data=json.dumps({"data":data}))
        print(response)
        result = response.json()
        print(result)
    except Exception as e:
        print(e)




kafka_consumer = ConsumerHandler(topic="group_id_log", group="nlp_fp",hosts=["10.13.40.138:9092"])

if __name__ == "__main__":
    # print offsets
    for topic_partiton in kafka_consumer.end_offsets:
        print(topic_partiton, "beginning_offsets:", kafka_consumer.beginning_offsets[topic_partiton],
                " end_offsets:" , kafka_consumer.end_offsets[topic_partiton])
    #exit()
    # 0: from beginning ; 1: from kafka offset ; 2: from ending
    kafka_method = 0
    # set offset
    kafka_consumer.seek(0,kafka_method)

    __stime = time.time()
    while True:
        # get msg from kafka
        data = kafka_consumer.read()
        if not data:
            print("empty")
            time.sleep(1)
            continue
        try:
            data = json.loads(data)
        except Exception as e:
            print(e)
            time.sleep(0.1)
            continue
        try:
            #print(data)
            ret = data_format(data)
            print(ret)
            fp(ret)
        except Exception as e:
            print(e)
            time.sleep(0.1)
            continue
        


